#include "KeywordFinder.h"
#include "../support/Epiplon.h"
#include "../support/Hasher.h"
#include "../support/Timer.h"
#include <string>
#include <map>
#include <list>
#include <iostream>

using namespace std;


KeywordFinder::KeywordFinder()
{
	t1=0;
	t2=0;
	t3=0;
	t4=0;
	t5=0;
	t6=0;
	t7=0;
	t8=0;
	t9=0;
	t10=0;
	t11=0;
	t12=0;
	cout << "Carregando tabela" << endl;
	epiplon.loadLinksToIDTable();
	cout << "Carregada!!!" << endl; 
}

KeywordFinder::~KeywordFinder()
{
}

void KeywordFinder::coutTimes() {
	cout << "finding quotes: 			" << t1 << endl;
	cout << "raw text and link: 			" << t2 << endl;
	cout << "limit text size:			" << t3 << endl;
	cout << "tolower all: 				" << t4 << endl;
	cout << "pre-normalizing link: 			" << t5 << endl;
	cout << "normalizing link: 			" << t9 << endl;
	cout << "text cleanup: 				" << t6 << endl;
	cout << "if grande: 				" << t11 << endl;
	cout << "retrieving text words: 			" << t7 << endl;
	cout << "pushing back pairs: 			" << t8 << endl;
	cout << "href fim: 				" << t12 << endl;
	cout << "app total: 				" << t10 << endl;
	
}

void KeywordFinder::findKeywords(string & htmlPage, string url, list<pair<unsigned long long, unsigned long long> > & myList) {
	uint href = htmlPage.find("href");
	Timer ti1;
	Timer ti2;
	Timer ti3;
	Timer ti4;
	Timer ti5;
	Timer ti6;
	Timer ti7;
	Timer ti8;
	Timer ti9;
	Timer ti10;
	Timer ti11;
	Timer ti12;
	ti10.begin();
	while (href != string::npos) {
		string link = "";
		string text = "";
		
		ti1.begin();
		
		uint quoteBegin = htmlPage.find("\"", href + 1);
		if (quoteBegin == string::npos)
			break;
		uint quoteEnd = htmlPage.find("\"", quoteBegin+1);
		if (quoteEnd == string::npos)
			break;
		ti1.end();
		t1 += ti1.diff();
		
		if (quoteEnd < quoteBegin + 300) {

			ti2.begin();
			
			link = htmlPage.substr(quoteBegin+1, quoteEnd - quoteBegin - 1);
			uint tagClose = htmlPage.find(">", quoteEnd);
			if (tagClose == string::npos)
				break;
			uint tagOpen = htmlPage.find("<", tagClose);
			if (tagOpen == string::npos)
				break;
			text = htmlPage.substr(tagClose + 1, tagOpen - tagClose - 1);
			

			ti2.end();
			t2 += ti2.diff();
			
			
			ti3.begin();
			if (text.size() > 120) {
				uint lastSpace = text.find_last_of(" ", 119);
				if (lastSpace != string::npos)
					text = text.erase(lastSpace);
				else
					text = text.erase(119);
			}
			ti3.end();
			t3 += ti3.diff();
			
			ti4.begin();
			urlNormalizer.toLower(text);
			urlNormalizer.toLower(link);
			ti4.end();
			t4 += ti4.diff();
//			cout << "antes link: " << link << endl;
			
			
			//tratando link
			ti5.begin();
			if (link.find("javascript") == string::npos) {
				uint protocolIndex = link.find("://");
				if (protocolIndex == string::npos) {
					urlNormalizer.removePath(url);
					uint dotIndex = link.find_first_of('/');
					if (dotIndex == 0)
						link = url + link;
					else
						link = url + "/" + link;
				}
				ti5.end();
				t5 += ti5.diff();
				ti9.begin();
				urlNormalizer.normalize(link);
				ti9.end();
				t9 += ti9.diff();
			}
			else {
				ti5.end();
				t5 += ti5.diff();
				link = "";
			}

			
			ti6.begin();
			text = substituter.convertHtmlString(text);
			
			urlNormalizer.removeEnters(text);
			ti6.end();
			t6 += ti6.diff();
			
			ti11.begin();
			if (link.find_first_not_of(" \t\n\r") != string::npos && text.find_first_not_of(" \t\n\r") != string::npos) {
//				cout << "Link: " << link << endl;
//				cout << "Texto: " << text << endl;

				ti11.end();
				t11 += ti11.diff();
				ti7.begin();
				
				uint id = epiplon.getIDFromLink( Hasher::abcHash(link) );
				list<string> keywordList;
				uint lastSpaceIndex = 0;
				uint spaceIndex = text.find_first_of(" ");
				while (spaceIndex != string::npos) {
					if (spaceIndex > lastSpaceIndex + 1) //soh palavras de pelo menos 2 caracteres
						keywordList.push_back( text.substr(lastSpaceIndex, spaceIndex - lastSpaceIndex) );
					lastSpaceIndex = spaceIndex + 1;
					spaceIndex = text.find_first_of(" ", lastSpaceIndex);
				}
				
				if (lastSpaceIndex == 0 && text.size() > 1) //soh palavras de pelo menos 2 caracteres
					keywordList.push_back(text);
				else if (text.size() > lastSpaceIndex + 1) //soh palavras de pelo menos 2 caracteres
					keywordList.push_back(text.substr(lastSpaceIndex, text.size() - lastSpaceIndex));

				ti7.end();
				t7 += ti7.diff();
				
				
				list<string>::iterator testIt;
//				cout << "Id: " << id << endl;
//				cout << "Palavras: ";
				ti8.begin();
				for (testIt = keywordList.begin(); testIt != keywordList.end(); testIt++) {
					myList.push_back( pair<unsigned long long, unsigned long long>(id, Hasher::abcHash(*testIt) ) );
//					cout << *testIt << " hash: " << Hasher::abcHash(*testIt) << ", ";
				}
				ti8.end();
				t8 += ti8.diff();
//				cout << endl;
				
				

				
			}
//			else
//				cout << "eh, nao foi dessa vez... " << text << endl;
			
			
			
			
		}
		ti12.begin();
		href = htmlPage.find("href", quoteEnd+1);
		ti12.end();
		t12 += ti12.diff();
	}
	
	ti10.end();
	t10 += ti10.diff();
}
